{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50000 entries, 0 to 49999\n",
      "Data columns (total 2 columns):\n",
      " #   Column   Non-Null Count  Dtype \n",
      "---  ------   --------------  ----- \n",
      " 0   label    50000 non-null  object\n",
      " 1   content  50000 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 781.4+ KB\n",
      "None\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10000 entries, 0 to 9999\n",
      "Data columns (total 2 columns):\n",
      " #   Column   Non-Null Count  Dtype \n",
      "---  ------   --------------  ----- \n",
      " 0   label    10000 non-null  object\n",
      " 1   content  10000 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 156.4+ KB\n",
      "None\n",
      "  label                                            content\n",
      "0    体育  马晓旭意外受伤让国奥警惕 无奈大雨格外青睐殷家军记者傅亚雨沈阳报道 来到沈阳，国奥队依然没有...\n",
      "1    体育  商瑞华首战复仇心切 中国玫瑰要用美国方式攻克瑞典多曼来了，瑞典来了，商瑞华首战求3分的信心也...\n",
      "2    体育  冠军球队迎新欢乐派对 黄旭获大奖张军赢下PK赛新浪体育讯12月27日晚，“冠军高尔夫球队迎新...\n",
      "3    体育  辽足签约危机引注册难关 高层威逼利诱合同笑里藏刀新浪体育讯2月24日，辽足爆发了集体拒签风波...\n",
      "4    体育  揭秘谢亚龙被带走：总局电话骗局 复制南杨轨迹体坛周报特约记者张锐北京报道  谢亚龙已经被公安...\n",
      "  label                                            content\n",
      "0    体育  鲍勃库西奖归谁属？ NCAA最强控卫是坎巴还是弗神新浪体育讯如今，本赛季的NCAA进入到了末...\n",
      "1    体育  麦基砍28+18+5却充满寂寞 纪录之夜他的痛阿联最懂新浪体育讯上天对每个人都是公平的，贾维...\n",
      "2    体育  黄蜂vs湖人首发：科比冲击七连胜 火箭两旧将登场新浪体育讯北京时间3月28日，NBA常规赛洛...\n",
      "3    体育  双面谢亚龙作秀终成做作 谁来为低劣行政能力埋单是谁任命了谢亚龙？谁放纵了谢亚龙？谁又该为谢亚...\n",
      "4    体育  兔年首战山西换帅后有虎胆 张学文用乔丹名言励志今晚客场挑战浙江稠州银行队，是山西汾酒男篮的兔...\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "# 数据加载\n",
    "train_df = pd.read_csv(\"../data/cnews/cnews.train.txt\", sep='\\t', names=['label', 'content'])\n",
    "test_df = pd.read_csv(\"../data/cnews/cnews.test.txt\", sep='\\t', names=['label', 'content'])\n",
    "print(train_df.info())\n",
    "print(test_df.info())\n",
    "print(train_df.head(5))\n",
    "print(test_df.head(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 中文分词，去掉停用词"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "import jieba\n",
    "\n",
    "X_train = train_df['content']\n",
    "y_train = train_df['label']\n",
    "X_test = test_df['content']\n",
    "y_test = test_df['label']\n",
    "\n",
    "\n",
    "# 数据预处理\n",
    "def cut_content(data):\n",
    "    \"\"\"\n",
    "    利用jieba工具进行中文分词\n",
    "    :param data: 数据\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    words = data.apply(lambda x: ' '.join(jieba.cut(x)))\n",
    "    return words\n",
    "\n",
    "\n",
    "# 加载停用词表\n",
    "stopwords_file = open('../data/cnews/cnews.vocab.txt', encoding='utf-8')\n",
    "stopwords_list = stopwords_file.readlines()\n",
    "stopwords = [x.strip() for x in stopwords_list]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## TF-IDF特征提取"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache C:\\Users\\shan\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.773 seconds.\n",
      "Prefix dict has been built successfully.\n",
      "D:\\Software\\Anaconda3\\envs\\nlp\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:396: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['PAD'] not in stop_words.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 5000)\n",
      "(10000, 5000)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "# TF-IDF提取特征\n",
    "tfidf_vector = TfidfVectorizer(stop_words=stopwords, max_features=5000, lowercase=False, sublinear_tf=True, max_df=0.8)\n",
    "tfidf_vector.fit(cut_content(X_train))\n",
    "X_train_tfidf = tfidf_vector.transform(cut_content(X_train))\n",
    "X_test_tfidf = tfidf_vector.transform(cut_content(X_test))\n",
    "print(X_train_tfidf.shape)  # (50000, 5000)\n",
    "print(X_test_tfidf.shape) # (10000, 5000)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 特征选择-卡方统计量"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 3000)\n",
      "(10000, 3000)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "\n",
    "# 利用卡方统计量进行特征选择\n",
    "selector = SelectKBest(chi2, k=3000)\n",
    "X_train_tfidf_chi = selector.fit_transform(X_train_tfidf, y_train)\n",
    "X_test_tfidf_chi = selector.transform(X_test_tfidf)\n",
    "print(X_train_tfidf_chi.shape)\n",
    "print(X_test_tfidf_chi.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 模型训练-NP"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "clf_nb = MultinomialNB(alpha=0.2)  # 模型参数可以根据分类结果进行调优\n",
    "# 使用TF-IDF作为特征向量\n",
    "clf_nb.fit(X_train_tfidf, y_train)  # 模型训练\n",
    "y_pred = clf_nb.predict(X_test_tfidf)  # 模型预测\n",
    "\n",
    "# 使用TF-IDF+chi作为特征向量\n",
    "clf_nb.fit(X_train_tfidf_chi, y_train)  # 模型训练\n",
    "y_pred_chi = clf_nb.predict(X_test_tfidf_chi)  # 模型预测"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 模型评估"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9126\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          体育       1.00      0.99      0.99      1000\n",
      "          娱乐       0.93      0.98      0.95      1000\n",
      "          家居       0.94      0.44      0.60      1000\n",
      "          房产       0.66      0.88      0.76      1000\n",
      "          教育       0.91      0.92      0.91      1000\n",
      "          时尚       0.98      0.96      0.97      1000\n",
      "          时政       0.91      0.93      0.92      1000\n",
      "          游戏       0.94      0.98      0.96      1000\n",
      "          科技       0.94      0.97      0.95      1000\n",
      "          财经       0.94      0.99      0.96      1000\n",
      "\n",
      "    accuracy                           0.90     10000\n",
      "   macro avg       0.91      0.90      0.90     10000\n",
      "weighted avg       0.91      0.90      0.90     10000\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          体育       1.00      0.99      1.00      1000\n",
      "          娱乐       0.95      0.99      0.97      1000\n",
      "          家居       0.94      0.48      0.63      1000\n",
      "          房产       0.68      0.90      0.78      1000\n",
      "          教育       0.92      0.93      0.93      1000\n",
      "          时尚       0.98      0.96      0.97      1000\n",
      "          时政       0.91      0.93      0.92      1000\n",
      "          游戏       0.95      0.98      0.97      1000\n",
      "          科技       0.93      0.98      0.95      1000\n",
      "          财经       0.95      0.99      0.97      1000\n",
      "\n",
      "    accuracy                           0.91     10000\n",
      "   macro avg       0.92      0.91      0.91     10000\n",
      "weighted avg       0.92      0.91      0.91     10000\n",
      "\n",
      "[[987   6   0   0   3   0   2   2   0   0]\n",
      " [  0 985   0   0   2   4   3   3   2   1]\n",
      " [  0  16 444 401  43   5  35  28  17  11]\n",
      " [  0   8  15 883  14   1  33   4   3  39]\n",
      " [  0  20   2  10 921   4  12   2  25   4]\n",
      " [  0  15   7   0   6 962   0   4   6   0]\n",
      " [  0   3   0  29  20   0 932   1   6   9]\n",
      " [  0  10   0   1   1   6   1 977   4   0]\n",
      " [  0   0   5   1   3   0   1  19 970   1]\n",
      " [  0   0   0   5   1   0   5   1   0 988]]\n",
      "[[991   2   0   0   2   0   4   1   0   0]\n",
      " [  0 988   0   1   2   4   1   1   2   1]\n",
      " [  0  11 479 377  29   5  41  25  25   8]\n",
      " [  0   6  15 900  10   0  31   2   3  33]\n",
      " [  0  14   1   9 928   2   9   2  30   5]\n",
      " [  0  15   9   0   6 958   0   4   8   0]\n",
      " [  0   3   0  28  22   0 929   1   7  10]\n",
      " [  0   5   1   1   2   5   2 981   3   0]\n",
      " [  0   0   4   2   2   1   0  12 979   0]\n",
      " [  0   0   0   2   2   0   3   0   0 993]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "# 测试集准确率\n",
    "# print(clf_nb.score(X_test_tfidf, y_test)) # 0.9177\n",
    "print(clf_nb.score(X_test_tfidf_chi, y_test))  # 0.9226\n",
    "\n",
    "# 查看各类指标\n",
    "print(classification_report(y_test, y_pred))\n",
    "print(classification_report(y_test, y_pred_chi))\n",
    "\n",
    "# 查看混淆矩阵\n",
    "print(confusion_matrix(y_test, y_pred))\n",
    "print(confusion_matrix(y_test, y_pred_chi))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}